'''
File: tm5Semmler.py
Author: Niklas Semmler
Description: Taalmodelen Assignment 5
'''
#!/usr/bin/python
from __future__ import division
import sys
from collections import defaultdict
import tm3Semmler 

class Ddict(): # by Parand Tony Darugar
    def __init__(self, default=None):
        self.default = default

    def __getitem__(self, key):
        if not self.has_key(key):
            self[key] = self.default()
        return dict.__getitem__(self, key)


def n_gram(text, n):
    """  create n-grams from word list
    """
    nGrams = Ddict(dict)
    nGrams['a']['b'] = 1
    for i in range(len(text)-2):
        if text[i] in nGrams:
            if text[i+1] in nGrams[text[i]]:
                nGrams[text[i]][text[i+1]] += 1
            else:
                nGrams[text[i]][text[i+1]] = 1 
        else:
            nGrams[text[i]][text[i+1]] = 1 
    return nGrams

def merge_Ddict(bigDict, smallDict):
    """ merges two 2D dicts. In case of any collsiding keys, the value of both is summed
    """
    for k in smallDict:
        if k in bigDict:
            for k2 in smallDict[k]:
                if k2 in bigDict[k]:
                    bigDict[k][k2] = smallDict[k][k2] + bigDict[k][k2]
                else:
                    bigDict[k][k2] = smallDict[k][k2]
        else:
            bigDict[k] = smallDict[k]
    return bigDict

def n_gram2(text, n):
    """  create n-grams from paragraphs
    """
    nGrams = {} 
    for paragraph in text:
        nGram = n_gram(paragraph, n)
        nGrams = merge_dict(nGrams, nGram)
    return nGrams

def freq_of_freq(ngrams, K):
    """ put all Nr values into a dict
    """
    nAll = 0
    nR = [0]*(K+2)      
    for k,v in ngrams.items():
        nAll += v
        if v <= K+1:
            nR[v] += 1
    return nR, nAll

def smooth_count(r, Nr, NrPlus, NkPlus, nOne, k):
    """ smooth the count to count* by Katz
    """
    numer = (r + 1) * (NrPlus/Nr) - r * (((k+1)*NkPlus)/nOne)
    denom = 1 - (((k+1) * NkPlus) / nOne)
    
    return numer / denom

def report_gap(nR, nKPlus, r):
    """ report gaps in the frequencies of frequencies
    """
    if not nR:
        print "No ngram has the frequency %s\nTo calculate smoothed counts you have either to implement linear regression or chose a smaller k." % (r)
    elif not nKPlus:  
        print "All smoothed counts have been set to 0, because the frequency k+1 did not occur. Please either implement linear regression or choose a lower k."

def smooth_freq(nR, K):
    """ smooth all freq with 1>=r<=k
        Assumption All frequencies up to k+1 appear in the ngrams at least once.

    """ 
    nOne = nR[1]
    nKPlus = nR[K+1]
    
    report_gap(1, nKPlus, -1)
    
    smoothedCount = [0] 

    for i in range(1,K+1):
        report_gap(nR[i], 1, i)
        smoothedCount.append(smooth_count(i, nR[i], nR[i+1], nKPlus, nOne, K))
    return smoothedCount

def calc_prob_sen2(testSet, nGrams):
    """ P(wi | wi-n+1, ..., wi-1)
    """
    results = []
    nWords = 0
    nPars = len(testSet)

    # multiply all bigrams
    for par in testSet:
        leng = len(par)
        nWords += leng
        newSentences = []
        
        prob = 1
        
        for i in range(0,leng-2):
            prob *= calc_prob_sen(par[i:i+2], nGrams)
    
    # multiply all results with probability of P(<Start>) to finish the deal
    pS = nPars / nWords  
    results = map(lambda x : x * pS, results) 
    return results

def main(argv=sys.argv):
    # CONSTANTS taken from the book
    K = 5
    n = 2 # -> bigrams

    trainFile = argv[1]
    testFile = argv[2]
    
    trainPar    = tm3Semmler.read_file(trainFile, '\n[\n]+')
    trainSigned = tm3Semmler.add_signs(trainPar, n)
    trainNGrams = n_gram(trainSigned, n)
    #nR, nAll = freq_of_freq(trainNGrams, K)
    #smoothedCount = smooth_freq(nR, K)
    #testPar = tm3Semmler.read_file(testFile, '\n[\n]+')
    #testSigned = tm3Semmler.add_signs(testPar, n)
    #testResult = calc_prob_sen2(testSigned, trainNGrams, nAll)
    #
    #print testSigned
    #
    #print nR 
    #print smoothedCount
    
    print trainNGrams
    #testPar    = read_file(testFile, '\n[\n]+')
    #testSigned = add_signs(testPar, n)
    return 0

if __name__ == '__main__':
    sys.exit(main())
